var request = require('request');
var utils = require('./utils.js');
var initUrl = "http://www.oschina.net";
var urlList = [initUrl];


// function parseHtmlGetUrl(html, url) {
//     var regEx = /<a\s+href="[^"]+"/gi;
//     var regEx2 = /<a\s+href="([^"]+)"/g;
//     if(!html) return [];
//     var allAnchor = html.match(regEx);
//     var urls = [];
//     if (allAnchor && allAnchor.length > 0) {
//         urls = allAnchor.map(function (item) {
//             var m = item.match(regEx2);
//             return utils.getFullUrl(url, m[1]);
//         });
//     }
//     return urls;
// }

function parseHtmlGetUrl(html, url) {
    var regEx2 = /<a\s+href="([^"]+)"/gi;
    if(!html) return [];
    var m;
    var urls=[];
    while(m=regEx2.exec(html)){
    	var rUrl = m[1];
    	var url = utils.getFullUrl(url,rUrl);
    	urls.push(url);
    }
    return urls;
}

function getHtmlFromUrl(url) {
    return function hello(cb) {
        request(url, function (err, res, body) {
            cb(err, body);
        });
    }
}


function* generator() {
    while (urlList.length > 0) {
        var url = urlList.pop();
        var result = yield getHtmlFromUrl(url);
        if (result.html) {
            var urls = parseHtmlGetUrl(result.html, url);
            // add ten
            var mid = urls.length / 2;
            var s = urls.slice(mid, mid + 3);// add 3
            console.log(s);
            console.log(urlList);
            urlList = urlList.concat(s);
        }
    }
}

function run() {
    var it = generator();
    function goNext(result) {
        if (result.done) return;
        var fn = result.value;
        fn(function (err, body) {
            var result = !!err ? {err: err} : {html: body};
            setTimeout(function () {
                goNext(it.next(result));
            }, 1000);
        });
    }
    goNext(it.next());
}


run();